package controller;

import crawler.MyCrawlerDetail;
import crawler.MyCrawlerUrlList;
import dao.CrawlerService;
import dao.CrawlerServiceImpl;
import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;
import org.apache.http.message.BasicHeader;
import vo.WareHouseUrls;

import java.util.*;

/**
 * 58同城
 */
public class CrawlerController58 {
    public static int MaxDepthOfCrawling = 1;//最大爬取深度，-1表示爬取深度无限制
    public static int numberOfCrawlers = 1;//爬虫线程数


    public static void main(String[] args) throws Exception {
        CrawlerController58 cont = new CrawlerController58();
//        cont.searchListUrl();//爬list
        cont.searchListDetailByUrl();//根据list爬detail
        cont.updateParseStatus();//改detail解析状态为1

    }
    /**
     * 根据爬取分页的listUr;爬取明细数据
     * */
    public void searchListDetailByUrl()throws Exception{
        CrawlController controller = CrawlerController58.getMyCrawlerController();
        CrawlerService service = new CrawlerServiceImpl();
        List<WareHouseUrls> list = service.urlListToParse();
        for (WareHouseUrls wareHouseUrls : list) {
            controller.addSeed(wareHouseUrls.getUrl());
        }
        controller.start(MyCrawlerDetail.class, numberOfCrawlers);
    }
    /**
     * 改变解析状态
     * */
    public void updateParseStatus(){
        //改变数据的parseStatus为1，目前操作是为0的数据，全查全改
        CrawlerService service = new CrawlerServiceImpl();
        service.updateUrlsStatus();
    }
    /**
     * 爬取分页的list数据（没单纯使用detail的爬取，是为了精确爬取和避免错误爬取）
     * */
    public void searchListUrl()throws Exception{
        CrawlController controller = CrawlerController58.getMyCrawlerController();
        //配置爬虫种子页面，就是规定的从哪里开始爬，可以配置多个种子页面
//        controller.addSeed("http://www.cdwl.gov.cn/index.php?m=content&c=index&a=lists&catid=20");
//        controller.addSeed("http://www.cdwl.gov.cn/index.php?m=content&c=index&a=lists&catid=30");
        controller.addSeed("http://cd.58.com/cangkucf/");
        controller.start(MyCrawlerUrlList.class, numberOfCrawlers);
    }
    //获取crawler controller
    public static CrawlController getMyCrawlerController() throws Exception {
        String crawlStorageFolder = "d://pachong";// 定义爬虫数据存储位置
        CrawlConfig config = new CrawlConfig();
        config.setMaxDepthOfCrawling(MaxDepthOfCrawling);
        config.setPolitenessDelay(200);// 对同一个host的两次request之间的间隔时间（ms），避免过快爬取
        config.setFollowRedirects(false);
        config.setCrawlStorageFolder(crawlStorageFolder);

        HashSet<BasicHeader> collections = new HashSet<BasicHeader>();
        collections.add(new BasicHeader("User-Agent","Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3192.0 Safari/537.36"));
        collections.add(new BasicHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"));
        collections.add(new BasicHeader("Accept-Encoding", "gzip,deflate,sdch"));
        collections.add(new BasicHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6"));
        collections.add(new BasicHeader("Content-Type","application/x-www-form-urlencoded;charset=UTF-8"));
        collections.add(new BasicHeader("Connection", "keep-alive"));
        collections.add(new BasicHeader("Cookie", "bid=fp-BlwmyeTY; __yadk_uid=dLpMqMsIGD1N38NzhbcG3E6QA33NQ9bE; ps=y; _pk_ref.100001.8cb4=%5B%22%22%2C%22%22%2C1506515077%2C%22https%3A%2F%2Faccounts.douban.com%2Flogin%3Falias%3D793890838%2540qq.com%26redir%3Dhttps%253A%252F%252Fwww.douban.com%26source%3DNone%26error%3D1013%22%5D; ll=\"108296\"; ue=\"793890838@qq.com\"; __utmt=1; _ga=GA1.2.388925103.1505404043; _gid=GA1.2.1409223546.1506515083; dbcl2=\"161927939:ZDwWtUnYaH4\"; ck=rMaO; ap=1; push_noty_num=0; push_doumail_num=0; __utma=30149280.388925103.1505404043.1506510959.1506515077.8; __utmb=30149280.22.9.1506516374528; __utmc=30149280; __utmz=30149280.1506510959.7.5.utmcsr=accounts.douban.com|utmccn=(referral)|utmcmd=referral|utmcct=/login; __utmv=30149280.16192; _pk_id.100001.8cb4=1df4f52fdf296b72.1505404042.8.1506516380.1506512502.; _pk_ses.100001.8cb4=*"));
        config.setDefaultHeaders(collections);

        PageFetcher pageFetcher = new PageFetcher(config);// 实例化页面获取器
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);

        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);
        return controller;
    }
}
